<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   Tumblr: Hashing Your Way To Handling 23,000 Blog Requests Per Second  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/436.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466461139: Accept with keywords: (title(0.0):社区,数盟, topn(0.233333333333):社区,联系,数盟,小象,深度学习,未分类,行业资讯,推荐系统,数据挖掘,图书,原作者,数据分析,人工智能,职位,职业规划,星环,档期,基础架构,文章,泰迪,可视化,数据,商业用途,投稿,合作伙伴,资源,编程语言,课程,评论,行业).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    Tumblr: Hashing Your Way To Handling 23,000 Blog Requests Per Second
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    Tumblr: Hashing Your Way To Handling 23,000 Blog Requests Per Second | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,115] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/436.html">
          Tumblr: Hashing Your Way To Handling 23,000 Blog Requests Per Second
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          2,406 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech" rel="category tag">
          文章
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        <em>
         This is a guest post by
         <a href="http://michael.tumblr.com/">
          Michael Schenck
         </a>
         , SRE Staff Engineer at Tumblr.
        </em>
       </p>
       <p>
        At Tumblr, blogs (or Tumblelog) are one of our most highly trafficked faces on the internet.  One of the most convenient aspects of tumblelogs is their highly cacheable nature, which is fantastic because of the high views/post ratio the Tumblr network offers our users.  That said, it’s not entirely trivial to scale out the perimeter proxy tier, let alone the caching tier, necessary for serving all of those requests.
       </p>
       <p>
        This article describes the architecture of the portion of our perimeter responsible for blogs serving, one of our more highly trafficked perimeter end-points.
       </p>
       <p>
        Here’s how we do it.
       </p>
       <h3>
        Stats
       </h3>
       <ul>
        <li>
         278 employees total, 6 on the team responsible for all of Tumblr’s perimeter (Perimeter-SRE), including one manager.
        </li>
        <li>
         Over 2800 servers, less than 20% are used for blog serving functionality
        </li>
        <li>
         Over 23,000 blog requests per second (at peak)
        </li>
        <li>
         Over 6,500 blog cache purge events per second (at peak)
        </li>
        <li>
         Over 196 million blogs
        </li>
        <li>
         Over 93 billion posts
        </li>
       </ul>
       <h3>
        Platform
       </h3>
       <ul>
        <li>
         <a href="http://www.haproxy.org/">
          HAProxy
         </a>
        </li>
        <li>
         <a href="https://www.varnish-cache.org/">
          Varnish
         </a>
        </li>
        <li>
         <a href="http://bird.network.cz/">
          Bird
         </a>
        </li>
       </ul>
       <h3>
        Where We Were – Map-Based Partitioning
       </h3>
       <p>
        In the early days, we only needed one active and one standby proxy server as well as varnish node. Both were very easy to manage, monitor, and make highly available.
       </p>
       <p>
        However, being inline to all user requests, capacity limits will be reached and next steps (even if not the ideal deployment) will need to be ready before you wind up with downtime due to popularity.
       </p>
       <h3>
        Outgrowing A Single Proxy Node
       </h3>
       <p>
        Outgrowing a single
        <i>
         active
        </i>
        proxy server is pretty common and often involves DNS. Something as basic as a round-robin A record might meet your needs, but often times it’s worth the extra money to go for a health-checking GSLB configuration (even if you only have one geographic location).
       </p>
       <p>
        The downside to DNS is that, while nameservers will respond with each IP at a fairly even rate, there is no guarantee that each lookup will be used for the same number of requests. User A might make 10 requests to a resolved IP in a single minute, where bot B might make 100 requests in the same minute. If you have two IPs, user A gets one and bot B gets the other and they’re the only two clients making requests, then one of your proxies will have 10X the request rate of the other.
       </p>
       <p>
        This effect can be mitigated with lower TTLs, such that a 30 second TTL can balance those 110 requests between the two proxies. For the first 30 seconds user A will go to proxy P1, and bot B will go to proxy P2. Then, their next resolution might swap the IPs, so that user A will send its requests to proxy P2 and bot B will send its requests to proxy P1. At the end of that minute window, each proxy would have handled roughly 60 requests. The downside to lower TTLs is more lookups, thus higher DNS costs (but DNS is typically one of your less expensive third-party services).
       </p>
       <h3>
        Outgrowing A Single Varnish Node
       </h3>
       <p>
        While DNS can buy you a lot of time with growing your proxy tier, scaling varnish is a little more complex. Even if your capacity limitation with a single varnish node revolves around request concurrency, simply adding two varnish nodes in round-robin is not what you want to do; this reduces cache-hit ratio, makes purging more resource/time intensive, and doesn’t actually increase the working size of your cache (only duplicates it).
       </p>
       <p>
        The simplest iteration to outgrowing a single varnish node is
        <i>
         static partitioning
        </i>
        . This involves determining your unique identifier, and to split this space between two varnish nodes.
       </p>
       <p>
        For Tumblelogs, this is the hostname of the blog. Since DNS is case-insensitive, you only have to worry about 40 characters; alphanumerics (a-z and 0-9) and four allowed characters (- _ . ~). So for two varnish nodes, blogs hostnames are split (on the first character) between these two cache nodes.
       </p>
       <h3>
        Evenly Distributed Partitioning – Through Consistent Hashing
       </h3>
       <p>
        Both of the previous examples (DNS round-robin and static partitioning) are steps in the right direction, but provide very coarse granularity in partitioning. At a small enough scale, this granularity isn’t necessarily problematic, but as your traffic starts to grow the variance becomes more significant. As a result, reducing the variance in traffic handled by your least-hot and most-hot nodes becomes more-and-more important. This is where consistent hashing can really shine.
       </p>
       <h3>
        Partitioning Proxy Traffic
       </h3>
       <p>
        If your servers are in an environment where you can influence the routing tables of the router(s) in-front of your servers, the routers between your users and your proxy servers, then you can take advantage of equal-cost multi-path routing (ECMP). ECMP affords you the ability to treat your proxies as slices in a consistent hash ring, then map requesters across these slices.
       </p>
       <p>
        This is accomplished by informing the routing infrastructure of multiple paths (proxy servers) to a particular destination IP (a highly available IP). ECMP will then hash the request source in order to determine which proxy should receive the packets for this request session. Typical ECMP implementations offer Layer 3 (IP-only) and Layer 3+4 (IP:port) hashing options. Layer 3 means that all requests from a particular IP will go to a particular proxy, which can be helpful for debugging but is imbalanced with large networks using a single NAT IP. Layer 3+4 typically provides the best distribution, but debugging particular clients becomes more challenging.
       </p>
       <p>
        There are a number of ways to
        <i>
         inform
        </i>
        the router(s) of multiple paths, but I suggest using OSPF or iBGP for dynamic route advertisements. One need only listen to the highly-available IP on a loopback interface, enable internal routing, and advertise one’s own IP as a next-hop to the highly available IP. We found that BIRD provides a light-weight and reliable means for performing route advertisements from our proxies.
       </p>
       <h3>
        Partitioning Varnish Traffic
       </h3>
       <p>
        Tumblelogs are identified by their fully qualified domain name (FQDN), such that all URI paths of a blog will always be found under that blogs FQDN. The majority of Tumblelogs are sub-domains of tumblr.com, such as
        <a href="http://engineering.tumblr.com/">
         engineering.tumblr.com
        </a>
        , however we also support users bringing their own custom domain names.
       </p>
       <p>
        When considering this variety of FQDNs, it’s clear that the TLD will have the least number of variations, then domain names (particularly due to the vast majority being tumblr.com), then sub-domains. So our most-significant bits appear at the leftmost positions of a variable-length string.
       </p>
       <h3>
        Understanding The Problem Domain
       </h3>
       <ul>
        <li>
         perfect – demonstrates if the hashing function was
         <i>
          perfect
         </i>
         when applied to our test dataset
        </li>
        <li>
         consistent_hdr – consistent hashing on Host header (best real-world results)
        </li>
        <li>
         consistent_hdr_use_domain_only – consistent hashing on base domain name (i.e. tumblr.com or foo.net), only two camps tumblr.com and all-others
        </li>
        <li>
         mapbased_firstchar – mapping Host header’s first character to varnish node (our original static partitioning implementation)
        </li>
        <li>
         mapbased_hdr – map based on Host header
        </li>
       </ul>
       <p>
        While consistent hashing was the clear front-runner for most-even distribution of tumblelog FQDNs, we then went on to determine if the hash function was appropriate. HAProxy uses the SDBM hashing function by default. However, after further investigation, comparing SDBM, CRC, MD5, DJB2, and so on, we determined that DJB2 offered even better distribution. This resulting in our submitting a patch to HAProxy to make the hash function configurable (see “Thanks” section for more information).
       </p>
       <h3>
        Comparing Static Partitioning To Consistent Hashing
       </h3>
       <p>
        This shows the change in variance between requests per second of each varnish node, before and after moving to the best-fit hash function.
       </p>
       <h3>
        Additional Considerations
       </h3>
       <h3>
        Node Growth
       </h3>
       <p>
        In either model, node growth will mean keyspace shift, thus cache invalidation. In the consistent hashing model, it’s much easier to predict the percent of keys that will be invalidated; essentially 1/N (N is the number of cache nodes prior to the new node being added). With the static partitioning model, unless you do analysis on the requests going to the node(s) you will be taking keyspace from, you’re left with the worst case of less-than or equal-to the total percent of keys on the nodes you’re taking keyspace from.
       </p>
       <h3>
        Node Failure
       </h3>
       <p>
        With static partitioning, a single node failure will result in 1/N keys being inaccessible unless you provide a fail-over option. HAProxy does allow you to have a standby node, but now you have a decision to make; do you have 2N cache nodes (one active, one standby) for each key space, or shared standby node(s). One extreme is a
        <i>
         waste
        </i>
        of 50% of your hardware, where the other end of the spectrum (1 standby node shared between all active nodes) means that two failed nodes results in the standby support 2X the keyspace of the other active nodes.
       </p>
       <p>
        With consistent hashing, node failures are handled automatically. When one node is removed, then 1/N keys are shifted (resulting in 1/N keys to be invalidated) and an even rise in keyspace per remaining active node.
       </p>
       <h3>
        Purging Cache
       </h3>
       <p>
        Sending purge requests to a single varnish node is easy, but purging from multiple varnish nodes should be just as easy. Instead of having to keep the proxies and purgers in sync, it’s easiest to just send all purge requests through the same proxies.
       </p>
       <p>
        It’s important to reject purge attempts from non-local IP space, as to prevent any malicious bulk purging.
       </p>
       <h3>
        Lessons Learned
       </h3>
       <ul>
        <li>
         <b>
          You know answers to questions you haven’t asked yet.
         </b>
         When facing a scaling challenge, don’t over look patterns you’re already using elsewhere.
        </li>
        <li>
         <b>
          Scale through simplicity.
         </b>
         Adding complexity to overcome scalability challenges may work in the short run, but will eventually catch up with you.
        </li>
        <li>
         <b>
          Know your hash function.
         </b>
         The hash function you use is just as important as deciding what to do with the hashes.
        </li>
        <li>
         <b>
          Degrade, not fail.
         </b>
         It’s advisable to have your proxies monitor their own ability to reach their backends. If they cannot, don’t stop advertising routes, just advertise non-preferential routes (such as a higher path cost, with OSPF). This way, if all backends become unhealthy, you can still serve error pages, instead of becoming unreachable.
        </li>
       </ul>
       <h3>
        Thanks
       </h3>
       <ul>
        <li>
         <a href="http://tumblr.mobocracy.net/">
          Blake Matheny
         </a>
         And
         <a href="http://andrew.tumblr.com/">
          Andrew Terng
         </a>
         For Their Hash Function Comparisons And Creating The Patch To HAProxy.
        </li>
        <li>
         <a href="http://maddalab.tumblr.com/">
          Bhaskar Maddala
         </a>
         For Working With The HAProxy Community To Get
         <a href="http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4.2-hash-type">
          This Functionality Added To The HAProxy 1.5 Release
         </a>
         .
        </li>
        <li>
         <a href="http://blog.arnoudvermeer.nl/">
          Arnoud Vermeer
         </a>
         And
         <a href="http://aaronprat.tumblr.com/">
          Aaron Prat
         </a>
         For Their Work With ECMP/OSPF Traffic Distribution.
        </li>
       </ul>
       <h2>
        Related Articles
       </h2>
       <ul>
        <li>
         <a href="https://news.ycombinator.com/item?id=8132473">
          On HackerNews
         </a>
        </li>
        <li>
         <a href="http://highscalability.com/blog/2013/5/20/the-tumblr-architecture-yahoo-bought-for-a-cool-billion-doll.html">
          The Tumblr Architecture Yahoo Bought For A Cool Billion Dollars
         </a>
        </li>
        <li>
         <a href="http://highscalability.com/blog/2012/2/13/tumblr-architecture-15-billion-page-views-a-month-and-harder.html">
          Tumblr Architecture – 15 Billion Page Views A Month And Harder To Scale Than Twitter
         </a>
        </li>
       </ul>
       <p>
        原文链接：http://highscalability.com/blog/2014/8/4/tumblr-hashing-your-way-to-handling-23000-blog-requests-per.html
       </p>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/24220.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/05/100910rrxp5plglxk4lcle-291x200.png"/>
        </a>
        <a class="link" href="http://dataunion.org/24220.html" rel="bookmark" title="520大数据：八成男性期待被表白 “套路”也是爱">
         520大数据：八成男性期待被表白 “套路”也是爱
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/23785.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/04/u7145986293872240263fm21gp0-298x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/23785.html" rel="bookmark" title="Docker安全基准：22项新增要点概述">
         Docker安全基准：22项新增要点概述
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/23703.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/04/1317020-300x163.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/23703.html" rel="bookmark" title="推荐：五个不容错过的数据科学家博客">
         推荐：五个不容错过的数据科学家博客
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/23507.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/110606wjpugdldig4idddn.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/23507.html" rel="bookmark" title="解密深度学习在智能推荐系统的实践与应用">
         解密深度学习在智能推荐系统的实践与应用
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F436.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/436.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/436.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>